"""Filter parallel corpus"""
import logging

import io

from utils import has_zh


def is_len_token(s, min_len, max_len):
    len_t = len(s.split())
    return min_len <= len_t <= max_len


def is_len_ratio_enzh(en, zh, ratio):
    len_en = len(en.split())
    len_zh = len(zh)

    return len_en <= ratio * len_zh and len_zh <= ratio * len_en


def is_len_ratio_en(en, ratio=0.5):
    len_c = len(en)
    len_en = 0
    for c in en:
        if c.isalpha():
            len_en += 1
    return float(len_en) / len_c >= ratio


fn = "d15m.tsv"
out_fn = "d15m-filtered.tsv"
in_f = io.open(fn, encoding="utf-8")
out_f = io.open(out_fn, "w", encoding="utf-8")

logger = logging.getLogger()
fh = logging.FileHandler("../filtered.log", encoding="utf-8", mode="a")
formatter = logging.Formatter("%(asctime)s - %(name)s-%(levelname)s %(message)s")
fh.setFormatter(formatter)
logger.addHandler(fh)
logger.setLevel(logging.DEBUG)

# print(is_en("It was a cold winter’s day that Sunday."))

total = 0
passed = 0
for line in in_f:
    total += 1
    if total % 100000 == 0:
        print(total, passed)
    line = line.strip()
    cols = line.split("\t")
    if len(cols) >= 2:
        src, tgt = cols[:2]
    else:
        # print("Filtered-n", line)
        logger.debug("Filtered-n: {}".format(line))
        continue

    # if not (is_lang(src, "en") and is_lang(tgt, "zh")):
    #     print("Filtered", line)
    # The length of EN text based on token
    if not is_len_token(src, 2, 100):
        # print("Filtered-len", line)
        logger.debug("Filtered-len: {}".format(line))
        continue

    # EN text has no Chinese character
    if has_zh(src):
        # print("Filtered-has-zh", line)
        logger.debug("Filtered-has-zh: {}".format(line))
        continue

    # The length ratio of EN and ZH text
    if not is_len_ratio_enzh(src, tgt, 5):
        # print("Filtered-len-enzh", line)
        logger.debug("Filtered-len-ration-enzh: {}".format(line))
        continue

    # EN text has enough English character
    if not is_len_ratio_en(src):
        # print("Filtered-len-en", line)
        logger.debug("Filtered-len-ratio-en: {}".format(line))
        continue

    passed += 1
    out_f.write(line + "\n")

print(total, passed)
